import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/DNN_1d_TFM'
TIME_STEPS=144 #1d
UNITS=22
DROPOUT=0.779
ACTIVATION='sigmoid'
OPTIMIZER='adam'
EPOCHS=85
BATCHSIZE=23
VALIDATIONSPLIT=0.2
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (2973, 144, 1) y_train shape: (2973,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(Dense(units=UNITS, input_shape=(X_train.shape[1], X_train.shape[2]), activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dropout(rate=DROPOUT))
model.add(Dense(X_train.shape[2],activation=ACTIVATION))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=[rmse,'mse'])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense (Dense) (None, 144, 22) 44 _________________________________________________________________ dense_1 (Dense) (None, 144, 16) 368 _________________________________________________________________ dropout (Dropout) (None, 144, 16) 0 _________________________________________________________________ dense_2 (Dense) (None, 144, 1) 17 ================================================================= Total params: 429 Trainable params: 429 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/85 104/104 [==============================] - 1s 5ms/step - loss: 0.8709 - rmse: 0.8770 - mse: 1.0559 - val_loss: 1.2225 - val_rmse: 1.2236 - val_mse: 1.7069 Epoch 2/85 104/104 [==============================] - 0s 3ms/step - loss: 0.8386 - rmse: 0.8502 - mse: 0.9948 - val_loss: 1.1088 - val_rmse: 1.1119 - val_mse: 1.4404 Epoch 3/85 104/104 [==============================] - 0s 3ms/step - loss: 0.8184 - rmse: 0.8374 - mse: 0.9606 - val_loss: 1.0203 - val_rmse: 1.0250 - val_mse: 1.2484 Epoch 4/85 104/104 [==============================] - 0s 3ms/step - loss: 0.8040 - rmse: 0.8284 - mse: 0.9370 - val_loss: 0.9664 - val_rmse: 0.9723 - val_mse: 1.1385 Epoch 5/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7958 - rmse: 0.8241 - mse: 0.9240 - val_loss: 0.9391 - val_rmse: 0.9456 - val_mse: 1.0850 Epoch 6/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7914 - rmse: 0.8226 - mse: 0.9171 - val_loss: 0.9246 - val_rmse: 0.9314 - val_mse: 1.0571 Epoch 7/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7851 - rmse: 0.8180 - mse: 0.9063 - val_loss: 0.9114 - val_rmse: 0.9185 - val_mse: 1.0321 Epoch 8/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7786 - rmse: 0.8136 - mse: 0.8958 - val_loss: 0.9047 - val_rmse: 0.9120 - val_mse: 1.0196 Epoch 9/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7766 - rmse: 0.8131 - mse: 0.8925 - val_loss: 0.9018 - val_rmse: 0.9093 - val_mse: 1.0142 Epoch 10/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7755 - rmse: 0.8130 - mse: 0.8910 - val_loss: 0.9002 - val_rmse: 0.9080 - val_mse: 1.0116 Epoch 11/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7738 - rmse: 0.8121 - mse: 0.8876 - val_loss: 0.8994 - val_rmse: 0.9074 - val_mse: 1.0102 Epoch 12/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7739 - rmse: 0.8128 - mse: 0.8882 - val_loss: 0.8988 - val_rmse: 0.9071 - val_mse: 1.0094 Epoch 13/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7733 - rmse: 0.8125 - mse: 0.8869 - val_loss: 0.8984 - val_rmse: 0.9068 - val_mse: 1.0087 Epoch 14/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7726 - rmse: 0.8121 - mse: 0.8858 - val_loss: 0.8981 - val_rmse: 0.9066 - val_mse: 1.0083 Epoch 15/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7724 - rmse: 0.8122 - mse: 0.8856 - val_loss: 0.8978 - val_rmse: 0.9064 - val_mse: 1.0079 Epoch 16/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7711 - rmse: 0.8113 - mse: 0.8830 - val_loss: 0.8975 - val_rmse: 0.9062 - val_mse: 1.0075 Epoch 17/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7710 - rmse: 0.8114 - mse: 0.8838 - val_loss: 0.8974 - val_rmse: 0.9062 - val_mse: 1.0073 Epoch 18/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7703 - rmse: 0.8108 - mse: 0.8820 - val_loss: 0.8972 - val_rmse: 0.9061 - val_mse: 1.0071 Epoch 19/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7701 - rmse: 0.8107 - mse: 0.8818 - val_loss: 0.8971 - val_rmse: 0.9060 - val_mse: 1.0069 Epoch 20/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7697 - rmse: 0.8102 - mse: 0.8807 - val_loss: 0.8971 - val_rmse: 0.9061 - val_mse: 1.0071 Epoch 21/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7693 - rmse: 0.8097 - mse: 0.8804 - val_loss: 0.8969 - val_rmse: 0.9060 - val_mse: 1.0067 Epoch 22/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7688 - rmse: 0.8093 - mse: 0.8790 - val_loss: 0.8968 - val_rmse: 0.9059 - val_mse: 1.0065 Epoch 23/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7683 - rmse: 0.8090 - mse: 0.8786 - val_loss: 0.8967 - val_rmse: 0.9058 - val_mse: 1.0064 Epoch 24/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7683 - rmse: 0.8091 - mse: 0.8792 - val_loss: 0.8967 - val_rmse: 0.9059 - val_mse: 1.0065 Epoch 25/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7676 - rmse: 0.8083 - mse: 0.8776 - val_loss: 0.8967 - val_rmse: 0.9059 - val_mse: 1.0064 Epoch 26/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7673 - rmse: 0.8081 - mse: 0.8770 - val_loss: 0.8966 - val_rmse: 0.9059 - val_mse: 1.0064 Epoch 27/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7668 - rmse: 0.8077 - mse: 0.8761 - val_loss: 0.8966 - val_rmse: 0.9059 - val_mse: 1.0064 Epoch 28/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7666 - rmse: 0.8075 - mse: 0.8758 - val_loss: 0.8966 - val_rmse: 0.9059 - val_mse: 1.0064 Epoch 29/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7658 - rmse: 0.8068 - mse: 0.8744 - val_loss: 0.8965 - val_rmse: 0.9059 - val_mse: 1.0064 Epoch 30/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7655 - rmse: 0.8065 - mse: 0.8744 - val_loss: 0.8965 - val_rmse: 0.9059 - val_mse: 1.0063 Epoch 31/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7652 - rmse: 0.8062 - mse: 0.8739 - val_loss: 0.8965 - val_rmse: 0.9059 - val_mse: 1.0063 Epoch 32/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7647 - rmse: 0.8057 - mse: 0.8730 - val_loss: 0.8964 - val_rmse: 0.9057 - val_mse: 1.0061 Epoch 33/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7642 - rmse: 0.8050 - mse: 0.8714 - val_loss: 0.8962 - val_rmse: 0.9056 - val_mse: 1.0058 Epoch 34/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7646 - rmse: 0.8056 - mse: 0.8727 - val_loss: 0.8962 - val_rmse: 0.9056 - val_mse: 1.0058 Epoch 35/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7641 - rmse: 0.8052 - mse: 0.8720 - val_loss: 0.8961 - val_rmse: 0.9055 - val_mse: 1.0057 Epoch 36/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7635 - rmse: 0.8047 - mse: 0.8715 - val_loss: 0.8961 - val_rmse: 0.9055 - val_mse: 1.0056 Epoch 37/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7634 - rmse: 0.8048 - mse: 0.8708 - val_loss: 0.8960 - val_rmse: 0.9054 - val_mse: 1.0055 Epoch 38/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7623 - rmse: 0.8038 - mse: 0.8689 - val_loss: 0.8959 - val_rmse: 0.9054 - val_mse: 1.0054 Epoch 39/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7625 - rmse: 0.8040 - mse: 0.8695 - val_loss: 0.8958 - val_rmse: 0.9052 - val_mse: 1.0051 Epoch 40/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7626 - rmse: 0.8041 - mse: 0.8705 - val_loss: 0.8957 - val_rmse: 0.9051 - val_mse: 1.0049 Epoch 41/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7628 - rmse: 0.8044 - mse: 0.8708 - val_loss: 0.8955 - val_rmse: 0.9049 - val_mse: 1.0046 Epoch 42/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7622 - rmse: 0.8039 - mse: 0.8703 - val_loss: 0.8954 - val_rmse: 0.9048 - val_mse: 1.0043 Epoch 43/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7618 - rmse: 0.8035 - mse: 0.8696 - val_loss: 0.8953 - val_rmse: 0.9047 - val_mse: 1.0042 Epoch 44/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7613 - rmse: 0.8031 - mse: 0.8690 - val_loss: 0.8952 - val_rmse: 0.9046 - val_mse: 1.0040 Epoch 45/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7613 - rmse: 0.8032 - mse: 0.8689 - val_loss: 0.8951 - val_rmse: 0.9045 - val_mse: 1.0039 Epoch 46/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7613 - rmse: 0.8035 - mse: 0.8692 - val_loss: 0.8950 - val_rmse: 0.9044 - val_mse: 1.0036 Epoch 47/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7607 - rmse: 0.8028 - mse: 0.8679 - val_loss: 0.8949 - val_rmse: 0.9044 - val_mse: 1.0036 Epoch 48/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7603 - rmse: 0.8026 - mse: 0.8672 - val_loss: 0.8949 - val_rmse: 0.9043 - val_mse: 1.0035 Epoch 49/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7607 - rmse: 0.8031 - mse: 0.8685 - val_loss: 0.8948 - val_rmse: 0.9042 - val_mse: 1.0033 Epoch 50/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7606 - rmse: 0.8032 - mse: 0.8687 - val_loss: 0.8946 - val_rmse: 0.9041 - val_mse: 1.0030 Epoch 51/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7600 - rmse: 0.8026 - mse: 0.8674 - val_loss: 0.8945 - val_rmse: 0.9039 - val_mse: 1.0028 Epoch 52/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7597 - rmse: 0.8024 - mse: 0.8673 - val_loss: 0.8944 - val_rmse: 0.9039 - val_mse: 1.0027 Epoch 53/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7596 - rmse: 0.8024 - mse: 0.8674 - val_loss: 0.8944 - val_rmse: 0.9038 - val_mse: 1.0026 Epoch 54/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7592 - rmse: 0.8022 - mse: 0.8666 - val_loss: 0.8943 - val_rmse: 0.9037 - val_mse: 1.0024 Epoch 55/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7589 - rmse: 0.8020 - mse: 0.8666 - val_loss: 0.8942 - val_rmse: 0.9037 - val_mse: 1.0023 Epoch 56/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7588 - rmse: 0.8020 - mse: 0.8665 - val_loss: 0.8942 - val_rmse: 0.9037 - val_mse: 1.0022 Epoch 57/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7593 - rmse: 0.8025 - mse: 0.8671 - val_loss: 0.8941 - val_rmse: 0.9036 - val_mse: 1.0021 Epoch 58/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7586 - rmse: 0.8018 - mse: 0.8652 - val_loss: 0.8940 - val_rmse: 0.9035 - val_mse: 1.0020 Epoch 59/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7589 - rmse: 0.8024 - mse: 0.8669 - val_loss: 0.8940 - val_rmse: 0.9035 - val_mse: 1.0019 Epoch 60/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7587 - rmse: 0.8022 - mse: 0.8668 - val_loss: 0.8939 - val_rmse: 0.9034 - val_mse: 1.0018 Epoch 61/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7584 - rmse: 0.8018 - mse: 0.8660 - val_loss: 0.8938 - val_rmse: 0.9033 - val_mse: 1.0016 Epoch 62/85 104/104 [==============================] - 0s 2ms/step - loss: 0.7579 - rmse: 0.8013 - mse: 0.8651 - val_loss: 0.8938 - val_rmse: 0.9033 - val_mse: 1.0016 Epoch 63/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7583 - rmse: 0.8019 - mse: 0.8661 - val_loss: 0.8937 - val_rmse: 0.9032 - val_mse: 1.0014 Epoch 64/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7579 - rmse: 0.8016 - mse: 0.8653 - val_loss: 0.8937 - val_rmse: 0.9032 - val_mse: 1.0015 Epoch 65/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7579 - rmse: 0.8017 - mse: 0.8650 - val_loss: 0.8936 - val_rmse: 0.9032 - val_mse: 1.0013 Epoch 66/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7577 - rmse: 0.8016 - mse: 0.8652 - val_loss: 0.8936 - val_rmse: 0.9032 - val_mse: 1.0013 Epoch 67/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7574 - rmse: 0.8014 - mse: 0.8647 - val_loss: 0.8935 - val_rmse: 0.9031 - val_mse: 1.0011 Epoch 68/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7581 - rmse: 0.8021 - mse: 0.8661 - val_loss: 0.8935 - val_rmse: 0.9030 - val_mse: 1.0011 Epoch 69/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7573 - rmse: 0.8014 - mse: 0.8651 - val_loss: 0.8934 - val_rmse: 0.9029 - val_mse: 1.0009 Epoch 70/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7578 - rmse: 0.8021 - mse: 0.8660 - val_loss: 0.8934 - val_rmse: 0.9029 - val_mse: 1.0009 Epoch 71/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7571 - rmse: 0.8013 - mse: 0.8645 - val_loss: 0.8932 - val_rmse: 0.9027 - val_mse: 1.0006 Epoch 72/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7571 - rmse: 0.8012 - mse: 0.8644 - val_loss: 0.8932 - val_rmse: 0.9027 - val_mse: 1.0005 Epoch 73/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7569 - rmse: 0.8012 - mse: 0.8648 - val_loss: 0.8932 - val_rmse: 0.9027 - val_mse: 1.0006 Epoch 74/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7571 - rmse: 0.8014 - mse: 0.8653 - val_loss: 0.8932 - val_rmse: 0.9028 - val_mse: 1.0006 Epoch 75/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7569 - rmse: 0.8013 - mse: 0.8649 - val_loss: 0.8931 - val_rmse: 0.9027 - val_mse: 1.0005 Epoch 76/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7571 - rmse: 0.8016 - mse: 0.8655 - val_loss: 0.8931 - val_rmse: 0.9026 - val_mse: 1.0004 Epoch 77/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7563 - rmse: 0.8008 - mse: 0.8643 - val_loss: 0.8930 - val_rmse: 0.9025 - val_mse: 1.0002 Epoch 78/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7569 - rmse: 0.8016 - mse: 0.8655 - val_loss: 0.8930 - val_rmse: 0.9025 - val_mse: 1.0002 Epoch 79/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7564 - rmse: 0.8011 - mse: 0.8642 - val_loss: 0.8930 - val_rmse: 0.9026 - val_mse: 1.0002 Epoch 80/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7564 - rmse: 0.8013 - mse: 0.8643 - val_loss: 0.8929 - val_rmse: 0.9025 - val_mse: 1.0001 Epoch 81/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7565 - rmse: 0.8012 - mse: 0.8650 - val_loss: 0.8929 - val_rmse: 0.9025 - val_mse: 1.0000 Epoch 82/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7560 - rmse: 0.8009 - mse: 0.8644 - val_loss: 0.8928 - val_rmse: 0.9023 - val_mse: 0.9998 Epoch 83/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7557 - rmse: 0.8007 - mse: 0.8643 - val_loss: 0.8928 - val_rmse: 0.9024 - val_mse: 0.9999 Epoch 84/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7563 - rmse: 0.8013 - mse: 0.8659 - val_loss: 0.8927 - val_rmse: 0.9023 - val_mse: 0.9997 Epoch 85/85 104/104 [==============================] - 0s 3ms/step - loss: 0.7561 - rmse: 0.8010 - mse: 0.8644 - val_loss: 0.8927 - val_rmse: 0.9023 - val_mse: 0.9997
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.5533 Root Mean Square Error: 0.7239 Mean Square Error: 0.5241
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.9 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (636, 144, 1) 20/20 [==============================] - 0s 2ms/step - loss: 0.7421 - rmse: 0.7962 - mse: 0.9443 evaluate: [0.7420758605003357, 0.7961631417274475, 0.9443148374557495]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
LSTM: Mean Absolute Error: 0.4875 Root Mean Square Error: 0.6564 Mean Square Error: 0.4309
anomalies: (67, 10)
###################################################### ####################### PM25 ########################### Testing shape: (636, 144, 1) 20/20 [==============================] - 0s 1ms/step - loss: 0.7747 - rmse: 0.8311 - mse: 1.0170 evaluate: [0.7746624946594238, 0.8310636281967163, 1.0169856548309326]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.5016 Root Mean Square Error: 0.6581 Mean Square Error: 0.4331
anomalies: (72, 10)
###################################################### ####################### PM10 ########################### Testing shape: (636, 144, 1) 20/20 [==============================] - 0s 1ms/step - loss: 0.8028 - rmse: 0.8607 - mse: 1.0690 evaluate: [0.8028368949890137, 0.8606773018836975, 1.0689876079559326]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.5220 Root Mean Square Error: 0.6574 Mean Square Error: 0.4322
anomalies: (12, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (636, 144, 1) 20/20 [==============================] - ETA: 0s - loss: 0.6673 - rmse: 0.6955 - mse: 0.54 - 0s 1ms/step - loss: 0.8045 - rmse: 0.8623 - mse: 0.9836 evaluate: [0.8045419454574585, 0.8623204231262207, 0.9836204051971436] LSTM:
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
Mean Absolute Error: 0.5291 Root Mean Square Error: 0.6585 Mean Square Error: 0.4336
anomalies: (47, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (636, 144, 1) 1/20 [>.............................] - ETA: 0s - loss: 0.6562 - rmse: 0.6854 - mse: 0.5308
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
20/20 [==============================] - 0s 1ms/step - loss: 0.7945 - rmse: 0.8520 - mse: 0.9667 evaluate: [0.7945126891136169, 0.851960301399231, 0.9666828513145447] LSTM: Mean Absolute Error: 0.5223 Root Mean Square Error: 0.6559 Mean Square Error: 0.4302
anomalies: (47, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (636, 144, 1) 20/20 [==============================] - 0s 1ms/step - loss: 0.7854 - rmse: 0.8441 - mse: 0.9661 evaluate: [0.7853648066520691, 0.844149112701416, 0.9660542607307434]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.5088 Root Mean Square Error: 0.6500 Mean Square Error: 0.4225
anomalies: (47, 10)
######################################################